{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shortest Path Experiments\n",
    "\n",
    "In this notebook we run experiments on learning to optimize over shortest path routes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import gurobipy as gp \n",
    "import numpy as np \n",
    "import networkx as nx\n",
    "import pickle \n",
    "import time\n",
    "\n",
    "from reweighted_mse_helpers import *\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from joblib import Parallel, delayed\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from scipy.sparse import coo_matrix,csr_matrix\n",
    "\n",
    "from models import LinearModel\n",
    "from sgd import SGDLearner\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Shortest Path Optimization Oracle\n",
    "\n",
    "We start by defining an oracle class to solve shortest path instances. This could probably be sped up by substituting an implementation of djikstra or other quick shortest path solvers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ShortestPathOracle(object):\n",
    "    \n",
    "    def __init__(self, graph_params,  quiet=True):\n",
    "        \n",
    "        nodes = graph_params['nodes']\n",
    "        sources = graph_params['sources']\n",
    "        destinations = graph_params['destinations']\n",
    "        start_node = graph_params['start_node']\n",
    "        end_node = graph_params['end_node']\n",
    "\n",
    "        \n",
    "        n_nodes = len(nodes)\n",
    "        n_edges = len(sources)\n",
    "        d_feasibleregion = len(sources)\n",
    "        \n",
    "        # Hard code sparse node-edge incidence matrix!\n",
    "        I_vec = np.hstack([sources,destinations])\n",
    "        J_vec = np.hstack([range(n_edges),range(n_edges)])\n",
    "        V_vec = np.hstack([-np.ones(n_edges), np.ones(n_edges)])\n",
    "        \n",
    "        #Construct constraint matrix\n",
    "        A_mat = coo_matrix((V_vec, (I_vec, J_vec)))\n",
    "        A_mat = csr_matrix(A_mat)\n",
    "        bvec = np.zeros(n_nodes)\n",
    "        bvec[start_node] = -1; bvec[end_node] = 1; \n",
    "        \n",
    "        A_mat = csr_matrix(A_mat)\n",
    "        bvec = np.zeros(n_nodes)\n",
    "        bvec[start_node] = -1; bvec[end_node] = 1; \n",
    "        self.m = gp.Model()\n",
    "        if quiet: self.m.setParam(\"OutputFlag\", 0)\n",
    "        self.w = self.m.addMVar(n_edges, lb= 0, ub = 1)\n",
    "        self.m.addConstrs(A_mat[i,:] @ self.w == bvec[i] for i in range(n_nodes) )\n",
    "        \n",
    "    def init_model(self,params):\n",
    "        pass\n",
    "    \n",
    "    def solve(self, c):\n",
    "        self.m.setObjective( c @ self.w, gp.GRB.MINIMIZE)\n",
    "        self.m.optimize()\n",
    "        z_ast = self.m.objVal\n",
    "        w_ast = np.asarray([self.w[i].X for i in range(len(c))]).flatten()\n",
    "        return [z_ast, w_ast]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the SPO paper: \n",
    "In the following set of experiments on the shortest path problem we described, we fix the number of features at p = 5 throughout and, as previously mentioned, use a 5 × 5 grid network, which implies that d = 40. Hence, in total there are pd = 200 parameters to estimate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{(0, 0): 0, (0, 1): 1, (0, 2): 2, (0, 3): 3, (0, 4): 4, (1, 0): 5, (1, 1): 6, (1, 2): 7, (1, 3): 8, (1, 4): 9, (2, 0): 10, (2, 1): 11, (2, 2): 12, (2, 3): 13, (2, 4): 14, (3, 0): 15, (3, 1): 16, (3, 2): 17, (3, 3): 18, (3, 4): 19, (4, 0): 20, (4, 1): 21, (4, 2): 22, (4, 3): 23, (4, 4): 24}\n"
     ]
    }
   ],
   "source": [
    "[sources, destinations, scalar_nodes, tuple_nodes] = convert_grid_to_list(5, 5)\n",
    "grid_dim = 5\n",
    "nodes = np.unique(list(set(sources).union(set(destinations))))\n",
    "d_feasibleregion = len(sources)\n",
    "start_node = scalar_nodes[(0,0)]\n",
    "end_node = scalar_nodes[(4,4)]\n",
    "\n",
    "#We'll save these parameters and pass them on to each worker to generate the oracle\n",
    "graph_params = {\n",
    "    'nodes': nodes,\n",
    "    'sources': sources,\n",
    "    'destinations': destinations,\n",
    "    'start_node': start_node,\n",
    "    'end_node': end_node\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Generator \n",
    "\n",
    "We follow the set-up of Elmachtoub & Grigas (2017) and generate data using a polynomial kernal with some noise."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Data Generating Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate Data\n",
    "p_features=5\n",
    "B_true = rand(1,0.5,size=(d_feasibleregion, p_features))\n",
    "polykernel_noise_half_width = 0.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment\n",
    "\n",
    "#### Experiment Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Amount of training data\n",
    "N_s = [100, 1000, 2000]\n",
    "\n",
    "#degrees of mis-specification\n",
    "polykernel_degree_vec = [1, 2, 4, 6, 8]\n",
    "n_misp=len(polykernel_degree_vec)\n",
    "\n",
    "#Task loss mixture weights\n",
    "n_wghts = 4\n",
    "mixture_weights = np.array([0.2,0.4,0.6,0.8])\n",
    "\n",
    "#number of repetitions of each trial\n",
    "n_reps = 8\n",
    "\n",
    "n_test = 10000\n",
    "n_holdout = 500"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize choice of regressor\n",
    "regressor = LinearRegression\n",
    "random_regr=False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n , 100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  8.3min\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   8 | elapsed:  8.5min remaining: 25.4min\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   8 | elapsed:  8.5min remaining: 14.1min\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   8 | elapsed:  8.5min remaining:  8.5min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   8 | elapsed:  8.6min remaining:  5.2min\n",
      "[Parallel(n_jobs=-1)]: Done   6 out of   8 | elapsed:  8.7min remaining:  2.9min\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed:  8.7min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed:  8.7min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n , 1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed: 17.8min\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   8 | elapsed: 17.9min remaining: 53.7min\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   8 | elapsed: 18.2min remaining: 30.3min\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   8 | elapsed: 18.3min remaining: 18.3min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   8 | elapsed: 18.5min remaining: 11.1min\n",
      "[Parallel(n_jobs=-1)]: Done   6 out of   8 | elapsed: 18.5min remaining:  6.2min\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 18.6min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 18.6min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n , 2000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed: 36.1min\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   8 | elapsed: 36.3min remaining: 109.0min\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   8 | elapsed: 36.4min remaining: 60.6min\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   8 | elapsed: 36.4min remaining: 36.4min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   8 | elapsed: 36.5min remaining: 21.9min\n",
      "[Parallel(n_jobs=-1)]: Done   6 out of   8 | elapsed: 36.5min remaining: 12.2min\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 36.7min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 36.7min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n , 100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed: 13.5min\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   8 | elapsed: 13.5min remaining: 40.6min\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   8 | elapsed: 13.6min remaining: 22.7min\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   8 | elapsed: 13.7min remaining: 13.7min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   8 | elapsed: 13.7min remaining:  8.2min\n",
      "[Parallel(n_jobs=-1)]: Done   6 out of   8 | elapsed: 13.7min remaining:  4.6min\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 13.8min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 13.8min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n , 1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed: 21.1min\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   8 | elapsed: 21.3min remaining: 63.9min\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   8 | elapsed: 21.3min remaining: 35.5min\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   8 | elapsed: 21.3min remaining: 21.3min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   8 | elapsed: 21.3min remaining: 12.8min\n",
      "[Parallel(n_jobs=-1)]: Done   6 out of   8 | elapsed: 21.5min remaining:  7.2min\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 21.5min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 21.5min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n , 2000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed: 31.8min\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   8 | elapsed: 32.0min remaining: 96.0min\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   8 | elapsed: 32.0min remaining: 53.4min\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   8 | elapsed: 32.1min remaining: 32.1min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   8 | elapsed: 32.1min remaining: 19.3min\n",
      "[Parallel(n_jobs=-1)]: Done   6 out of   8 | elapsed: 32.1min remaining: 10.7min\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 32.3min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 32.3min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n , 100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  9.3min\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   8 | elapsed:  9.3min remaining: 28.0min\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   8 | elapsed:  9.4min remaining: 15.7min\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   8 | elapsed:  9.4min remaining:  9.4min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   8 | elapsed:  9.5min remaining:  5.7min\n",
      "[Parallel(n_jobs=-1)]: Done   6 out of   8 | elapsed:  9.5min remaining:  3.2min\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed:  9.6min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed:  9.6min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n , 1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed: 11.4min\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   8 | elapsed: 11.5min remaining: 34.6min\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   8 | elapsed: 11.6min remaining: 19.3min\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   8 | elapsed: 11.6min remaining: 11.6min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   8 | elapsed: 11.6min remaining:  7.0min\n",
      "[Parallel(n_jobs=-1)]: Done   6 out of   8 | elapsed: 11.6min remaining:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 11.7min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 11.7min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n , 2000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed: 16.5min\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   8 | elapsed: 16.6min remaining: 49.8min\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   8 | elapsed: 16.6min remaining: 27.7min\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   8 | elapsed: 16.8min remaining: 16.8min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   8 | elapsed: 16.8min remaining: 10.1min\n",
      "[Parallel(n_jobs=-1)]: Done   6 out of   8 | elapsed: 16.8min remaining:  5.6min\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 16.9min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 16.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n , 100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  7.0min\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   8 | elapsed:  7.1min remaining: 21.2min\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   8 | elapsed:  7.1min remaining: 11.8min\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   8 | elapsed:  7.2min remaining:  7.2min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   8 | elapsed:  7.2min remaining:  4.3min\n",
      "[Parallel(n_jobs=-1)]: Done   6 out of   8 | elapsed:  7.2min remaining:  2.4min\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed:  7.2min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed:  7.2min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n , 1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed: 11.5min\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   8 | elapsed: 11.6min remaining: 34.8min\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   8 | elapsed: 11.7min remaining: 19.4min\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   8 | elapsed: 11.7min remaining: 11.7min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   8 | elapsed: 11.7min remaining:  7.0min\n",
      "[Parallel(n_jobs=-1)]: Done   6 out of   8 | elapsed: 11.7min remaining:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 11.7min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 11.7min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n , 2000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed: 16.4min\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   8 | elapsed: 16.6min remaining: 49.9min\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   8 | elapsed: 16.6min remaining: 27.7min\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   8 | elapsed: 16.7min remaining: 16.7min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   8 | elapsed: 16.7min remaining: 10.0min\n",
      "[Parallel(n_jobs=-1)]: Done   6 out of   8 | elapsed: 16.8min remaining:  5.6min\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 16.8min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 16.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n , 100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  7.0min\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   8 | elapsed:  7.0min remaining: 21.0min\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   8 | elapsed:  7.1min remaining: 11.9min\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   8 | elapsed:  7.1min remaining:  7.1min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   8 | elapsed:  7.1min remaining:  4.3min\n",
      "[Parallel(n_jobs=-1)]: Done   6 out of   8 | elapsed:  7.2min remaining:  2.4min\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed:  7.2min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed:  7.2min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n , 1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed: 11.4min\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   8 | elapsed: 11.4min remaining: 34.2min\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   8 | elapsed: 11.6min remaining: 19.3min\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   8 | elapsed: 11.6min remaining: 11.6min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   8 | elapsed: 11.6min remaining:  7.0min\n",
      "[Parallel(n_jobs=-1)]: Done   6 out of   8 | elapsed: 11.6min remaining:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 11.7min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 11.7min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n , 2000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed: 16.5min\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   8 | elapsed: 16.5min remaining: 49.4min\n",
      "[Parallel(n_jobs=-1)]: Done   3 out of   8 | elapsed: 16.6min remaining: 27.7min\n",
      "[Parallel(n_jobs=-1)]: Done   4 out of   8 | elapsed: 16.7min remaining: 16.7min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   8 | elapsed: 16.7min remaining: 10.0min\n",
      "[Parallel(n_jobs=-1)]: Done   6 out of   8 | elapsed: 16.7min remaining:  5.6min\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 16.8min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed: 16.8min finished\n"
     ]
    }
   ],
   "source": [
    "results_df = pd.DataFrame()\n",
    "\n",
    "oracle = ShortestPathOracle(graph_params)\n",
    "\n",
    "\n",
    "for misspec_ind, degree in enumerate(polykernel_degree_vec): \n",
    "    \n",
    "    polykernel_degree = degree\n",
    "    data_params = [1000, n_test, n_holdout, polykernel_degree,polykernel_noise_half_width,B_true]\n",
    "    [X_train, c_train,X_validation, c_validation, X_test, c_test] = generate_data(1000, \n",
    "            n_test, n_holdout, polykernel_degree,polykernel_noise_half_width,B_true,gen_test=True)\n",
    "        \n",
    "    testDict = generateInstanceDict(X_test, c_test, oracle)\n",
    "\n",
    "    for ind_n, en_train in enumerate(N_s): \n",
    "        print('n ,', en_train)\n",
    "        data_params = [en_train, n_test, n_holdout, polykernel_degree,polykernel_noise_half_width,B_true]\n",
    "\n",
    "        # Each replication returns [original_regrets, reweighted_mean_regrets (n_wghts) ]\n",
    "        res = Parallel(n_jobs=-1, verbose=20)(delayed(run_replication_over_weights)(data_params, X_test, c_test, testDict, \n",
    "                                 mixture_weights, regressor, graph_params, num_reweights = 3, random_regr=False) for k in range(n_reps))\n",
    "        \n",
    "        res_new =  pd.DataFrame.from_records(np.array(res).flatten())\n",
    "        results_df = results_df.append(res_new)\n",
    "        results_df.to_csv('results/may19_exp_3.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
